{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warming up PyWSD (takes ~10 secs)... took 6.695136785507202 secs.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import time\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from nltk.corpus import wordnet as wn\n",
    "\n",
    "from pywsd.utils import lemmatize\n",
    "from pywsd.lesk import synset_signatures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "took 38.222004890441895\n"
     ]
    }
   ],
   "source": [
    "all_signatures = []\n",
    "\n",
    "start = time.time()\n",
    "for ss in wn.all_synsets():\n",
    "    ss_signature = {}\n",
    "    offset = ss.offset()\n",
    "    pos = ss.pos()\n",
    "    idx = str(offset).zfill(8) + '-' + pos\n",
    "    \n",
    "    ss_signature['simple'] = synset_signatures(ss, hyperhypo=True, adapted=False,\n",
    "                                               remove_stopwords=True, \n",
    "                                               to_lemmatize=True, remove_numbers=True,\n",
    "                                               lowercase=True)\n",
    "    \n",
    "    ss_signature['adapted'] = synset_signatures(ss, hyperhypo=True, adapted=True,\n",
    "                                                   remove_stopwords=True, \n",
    "                                                   to_lemmatize=True, remove_numbers=True,\n",
    "                                                   lowercase=True)\n",
    "    \n",
    "    ss_signature['original'] = synset_signatures(ss, original_lesk=True,\n",
    "                                                 remove_stopwords=True, \n",
    "                                                 to_lemmatize=True, remove_numbers=True,\n",
    "                                                 lowercase=True)\n",
    "    \n",
    "    \n",
    "    all_signatures.append({'name': ss.name(), 'offset-pos': idx, \n",
    "                           'original': ss_signature['original'], \n",
    "                           'simple': ss_signature['simple'], \n",
    "                           'adapted':ss_signature['adapted']})\n",
    "\n",
    "print('took {}'.format(time.time() - start), file=sys.stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>adapted</th>\n",
       "      <th>name</th>\n",
       "      <th>offset-pos</th>\n",
       "      <th>original</th>\n",
       "      <th>simple</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{know-how, project, able, computer, car, somet...</td>\n",
       "      <td>able.a.01</td>\n",
       "      <td>00001740-a</td>\n",
       "      <td>{having, know-how, by, to, (, something, neces...</td>\n",
       "      <td>{know-how, project, able, computer, car, somet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{know-how, without, car, necessary, usually, t...</td>\n",
       "      <td>unable.a.01</td>\n",
       "      <td>00002098-a</td>\n",
       "      <td>{having, know-how, not, by, to, (, necessary, ...</td>\n",
       "      <td>{know-how, without, car, necessary, usually, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{abaxial, side, leaf, away, axis, face, stem, ...</td>\n",
       "      <td>abaxial.a.01</td>\n",
       "      <td>00002312-a</td>\n",
       "      <td>{an, facing, from, away, axis, organism, or, t...</td>\n",
       "      <td>{abaxial, side, leaf, away, axis, face, stem, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{upper, know, side, leaf, axis, face, adaxial,...</td>\n",
       "      <td>adaxial.a.01</td>\n",
       "      <td>00002527-a</td>\n",
       "      <td>{an, facing, axis, to, nearest, organism, towa...</td>\n",
       "      <td>{upper, know, side, leaf, axis, face, adaxial,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{side, acroscopic, face, apex, toward}</td>\n",
       "      <td>acroscopic.a.01</td>\n",
       "      <td>00002730-a</td>\n",
       "      <td>{facing, side, apex, toward, on, or, the}</td>\n",
       "      <td>{side, acroscopic, face, apex, toward}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             adapted             name  \\\n",
       "0  {know-how, project, able, computer, car, somet...        able.a.01   \n",
       "1  {know-how, without, car, necessary, usually, t...      unable.a.01   \n",
       "2  {abaxial, side, leaf, away, axis, face, stem, ...     abaxial.a.01   \n",
       "3  {upper, know, side, leaf, axis, face, adaxial,...     adaxial.a.01   \n",
       "4             {side, acroscopic, face, apex, toward}  acroscopic.a.01   \n",
       "\n",
       "   offset-pos                                           original  \\\n",
       "0  00001740-a  {having, know-how, by, to, (, something, neces...   \n",
       "1  00002098-a  {having, know-how, not, by, to, (, necessary, ...   \n",
       "2  00002312-a  {an, facing, from, away, axis, organism, or, t...   \n",
       "3  00002527-a  {an, facing, axis, to, nearest, organism, towa...   \n",
       "4  00002730-a          {facing, side, apex, toward, on, or, the}   \n",
       "\n",
       "                                              simple  \n",
       "0  {know-how, project, able, computer, car, somet...  \n",
       "1  {know-how, without, car, necessary, usually, t...  \n",
       "2  {abaxial, side, leaf, away, axis, face, stem, ...  \n",
       "3  {upper, know, side, leaf, axis, face, adaxial,...  \n",
       "4             {side, acroscopic, face, apex, toward}  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(all_signatures)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>name</th>\n",
       "      <th>able.a.01</th>\n",
       "      <th>unable.a.01</th>\n",
       "      <th>abaxial.a.01</th>\n",
       "      <th>adaxial.a.01</th>\n",
       "      <th>acroscopic.a.01</th>\n",
       "      <th>basiscopic.a.01</th>\n",
       "      <th>abducent.a.01</th>\n",
       "      <th>adducent.a.01</th>\n",
       "      <th>nascent.a.01</th>\n",
       "      <th>emergent.s.02</th>\n",
       "      <th>...</th>\n",
       "      <th>overcast.v.01</th>\n",
       "      <th>overcloud.v.01</th>\n",
       "      <th>clear_up.v.04</th>\n",
       "      <th>blight.v.01</th>\n",
       "      <th>swamp.v.01</th>\n",
       "      <th>run_dry.v.01</th>\n",
       "      <th>fog_up.v.01</th>\n",
       "      <th>char.v.01</th>\n",
       "      <th>haze.v.01</th>\n",
       "      <th>deflagrate.v.01</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>adapted</th>\n",
       "      <td>{know-how, project, able, computer, car, somet...</td>\n",
       "      <td>{know-how, without, car, necessary, usually, t...</td>\n",
       "      <td>{abaxial, side, leaf, away, axis, face, stem, ...</td>\n",
       "      <td>{upper, know, side, leaf, axis, face, adaxial,...</td>\n",
       "      <td>{side, acroscopic, face, apex, toward}</td>\n",
       "      <td>{side, face, basiscopic, toward, base}</td>\n",
       "      <td>{away, draw, abduct, midline, muscle, part, bo...</td>\n",
       "      <td>{together, bring, draw, midline, toward, muscl...</td>\n",
       "      <td>{born, emergent, begin, emerge, chick, insurge...</td>\n",
       "      <td>{emergent, emerge, existence, republic, nascen...</td>\n",
       "      <td>...</td>\n",
       "      <td>{make, darken, often, fog_up, fall, weather, h...</td>\n",
       "      <td>{cloud_up, overcloud, cloud_over, darken, cove...</td>\n",
       "      <td>{storm, brighten, sky, light_up, become, clear...</td>\n",
       "      <td>{rain, may, smite, afflict, cause, garden, bli...</td>\n",
       "      <td>{submerge, swamp, every, tsunami, harbor, boat...</td>\n",
       "      <td>{dry_out, run_dry, summer, river, empty, water...</td>\n",
       "      <td>{overcast, fog_up, foggy, fog, windshield, clo...</td>\n",
       "      <td>{coal, charcoal, forest, without, fire, everyt...</td>\n",
       "      <td>{dull, haze, cloud, become, overcast, hazy, cl...</td>\n",
       "      <td>{exercise, deflagrate, burn, substance, rapidl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>offset-pos</th>\n",
       "      <td>00001740-a</td>\n",
       "      <td>00002098-a</td>\n",
       "      <td>00002312-a</td>\n",
       "      <td>00002527-a</td>\n",
       "      <td>00002730-a</td>\n",
       "      <td>00002843-a</td>\n",
       "      <td>00002956-a</td>\n",
       "      <td>00003131-a</td>\n",
       "      <td>00003356-a</td>\n",
       "      <td>00003553-s</td>\n",
       "      <td>...</td>\n",
       "      <td>02770717-v</td>\n",
       "      <td>02771020-v</td>\n",
       "      <td>02771169-v</td>\n",
       "      <td>02771320-v</td>\n",
       "      <td>02771564-v</td>\n",
       "      <td>02771756-v</td>\n",
       "      <td>02771888-v</td>\n",
       "      <td>02771997-v</td>\n",
       "      <td>02772202-v</td>\n",
       "      <td>02772310-v</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>original</th>\n",
       "      <td>{having, know-how, by, to, (, something, neces...</td>\n",
       "      <td>{having, know-how, not, by, to, (, necessary, ...</td>\n",
       "      <td>{an, facing, from, away, axis, organism, or, t...</td>\n",
       "      <td>{an, facing, axis, to, nearest, organism, towa...</td>\n",
       "      <td>{facing, side, apex, toward, on, or, the}</td>\n",
       "      <td>{facing, side, toward, on, base, or, the}</td>\n",
       "      <td>{an, from, away, drawing, ;, the, midline, or,...</td>\n",
       "      <td>{an, together, drawing, ;, the, midline, bring...</td>\n",
       "      <td>{being, beginning, or, born}</td>\n",
       "      <td>{into, existence, coming}</td>\n",
       "      <td>...</td>\n",
       "      <td>{overcast, cloudy, make, or}</td>\n",
       "      <td>{covered, become, with, clouds}</td>\n",
       "      <td>{become, clear}</td>\n",
       "      <td>{a, to, cause, blight, suffer}</td>\n",
       "      <td>{submerged, submerge, drenched, drench, or, be}</td>\n",
       "      <td>{empty, water, become, of}</td>\n",
       "      <td>{foggy, get}</td>\n",
       "      <td>{charcoal, burn, to}</td>\n",
       "      <td>{dull, or, become, hazy, cloudy, ,}</td>\n",
       "      <td>{to, and, with, burn, great, rapidly, intensit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>simple</th>\n",
       "      <td>{know-how, project, able, computer, car, somet...</td>\n",
       "      <td>{know-how, without, car, necessary, usually, t...</td>\n",
       "      <td>{abaxial, side, leaf, away, axis, face, stem, ...</td>\n",
       "      <td>{upper, know, side, leaf, axis, face, adaxial,...</td>\n",
       "      <td>{side, acroscopic, face, apex, toward}</td>\n",
       "      <td>{side, face, basiscopic, toward, base}</td>\n",
       "      <td>{away, draw, abduct, midline, muscle, part, bo...</td>\n",
       "      <td>{together, bring, draw, midline, toward, muscl...</td>\n",
       "      <td>{begin, born, chick, insurgency, nascent}</td>\n",
       "      <td>{emergent, emerge, existence, republic, come}</td>\n",
       "      <td>...</td>\n",
       "      <td>{make, darken, often, fog_up, fall, weather, h...</td>\n",
       "      <td>{cloud_up, overcloud, cloud_over, darken, cove...</td>\n",
       "      <td>{storm, brighten, sky, light_up, become, clear...</td>\n",
       "      <td>{rain, may, smite, afflict, cause, garden, bli...</td>\n",
       "      <td>{submerge, swamp, every, tsunami, harbor, boat...</td>\n",
       "      <td>{dry_out, run_dry, summer, river, empty, water...</td>\n",
       "      <td>{overcast, fog_up, foggy, fog, windshield, clo...</td>\n",
       "      <td>{coal, charcoal, forest, without, fire, everyt...</td>\n",
       "      <td>{dull, haze, cloud, become, overcast, hazy, cl...</td>\n",
       "      <td>{exercise, deflagrate, burn, substance, rapidl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 117659 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "name                                                able.a.01  \\\n",
       "adapted     {know-how, project, able, computer, car, somet...   \n",
       "offset-pos                                         00001740-a   \n",
       "original    {having, know-how, by, to, (, something, neces...   \n",
       "simple      {know-how, project, able, computer, car, somet...   \n",
       "\n",
       "name                                              unable.a.01  \\\n",
       "adapted     {know-how, without, car, necessary, usually, t...   \n",
       "offset-pos                                         00002098-a   \n",
       "original    {having, know-how, not, by, to, (, necessary, ...   \n",
       "simple      {know-how, without, car, necessary, usually, t...   \n",
       "\n",
       "name                                             abaxial.a.01  \\\n",
       "adapted     {abaxial, side, leaf, away, axis, face, stem, ...   \n",
       "offset-pos                                         00002312-a   \n",
       "original    {an, facing, from, away, axis, organism, or, t...   \n",
       "simple      {abaxial, side, leaf, away, axis, face, stem, ...   \n",
       "\n",
       "name                                             adaxial.a.01  \\\n",
       "adapted     {upper, know, side, leaf, axis, face, adaxial,...   \n",
       "offset-pos                                         00002527-a   \n",
       "original    {an, facing, axis, to, nearest, organism, towa...   \n",
       "simple      {upper, know, side, leaf, axis, face, adaxial,...   \n",
       "\n",
       "name                                  acroscopic.a.01  \\\n",
       "adapted        {side, acroscopic, face, apex, toward}   \n",
       "offset-pos                                 00002730-a   \n",
       "original    {facing, side, apex, toward, on, or, the}   \n",
       "simple         {side, acroscopic, face, apex, toward}   \n",
       "\n",
       "name                                  basiscopic.a.01  \\\n",
       "adapted        {side, face, basiscopic, toward, base}   \n",
       "offset-pos                                 00002843-a   \n",
       "original    {facing, side, toward, on, base, or, the}   \n",
       "simple         {side, face, basiscopic, toward, base}   \n",
       "\n",
       "name                                            abducent.a.01  \\\n",
       "adapted     {away, draw, abduct, midline, muscle, part, bo...   \n",
       "offset-pos                                         00002956-a   \n",
       "original    {an, from, away, drawing, ;, the, midline, or,...   \n",
       "simple      {away, draw, abduct, midline, muscle, part, bo...   \n",
       "\n",
       "name                                            adducent.a.01  \\\n",
       "adapted     {together, bring, draw, midline, toward, muscl...   \n",
       "offset-pos                                         00003131-a   \n",
       "original    {an, together, drawing, ;, the, midline, bring...   \n",
       "simple      {together, bring, draw, midline, toward, muscl...   \n",
       "\n",
       "name                                             nascent.a.01  \\\n",
       "adapted     {born, emergent, begin, emerge, chick, insurge...   \n",
       "offset-pos                                         00003356-a   \n",
       "original                         {being, beginning, or, born}   \n",
       "simple              {begin, born, chick, insurgency, nascent}   \n",
       "\n",
       "name                                            emergent.s.02  \\\n",
       "adapted     {emergent, emerge, existence, republic, nascen...   \n",
       "offset-pos                                         00003553-s   \n",
       "original                            {into, existence, coming}   \n",
       "simple          {emergent, emerge, existence, republic, come}   \n",
       "\n",
       "name                              ...                          \\\n",
       "adapted                           ...                           \n",
       "offset-pos                        ...                           \n",
       "original                          ...                           \n",
       "simple                            ...                           \n",
       "\n",
       "name                                            overcast.v.01  \\\n",
       "adapted     {make, darken, often, fog_up, fall, weather, h...   \n",
       "offset-pos                                         02770717-v   \n",
       "original                         {overcast, cloudy, make, or}   \n",
       "simple      {make, darken, often, fog_up, fall, weather, h...   \n",
       "\n",
       "name                                           overcloud.v.01  \\\n",
       "adapted     {cloud_up, overcloud, cloud_over, darken, cove...   \n",
       "offset-pos                                         02771020-v   \n",
       "original                      {covered, become, with, clouds}   \n",
       "simple      {cloud_up, overcloud, cloud_over, darken, cove...   \n",
       "\n",
       "name                                            clear_up.v.04  \\\n",
       "adapted     {storm, brighten, sky, light_up, become, clear...   \n",
       "offset-pos                                         02771169-v   \n",
       "original                                      {become, clear}   \n",
       "simple      {storm, brighten, sky, light_up, become, clear...   \n",
       "\n",
       "name                                              blight.v.01  \\\n",
       "adapted     {rain, may, smite, afflict, cause, garden, bli...   \n",
       "offset-pos                                         02771320-v   \n",
       "original                       {a, to, cause, blight, suffer}   \n",
       "simple      {rain, may, smite, afflict, cause, garden, bli...   \n",
       "\n",
       "name                                               swamp.v.01  \\\n",
       "adapted     {submerge, swamp, every, tsunami, harbor, boat...   \n",
       "offset-pos                                         02771564-v   \n",
       "original      {submerged, submerge, drenched, drench, or, be}   \n",
       "simple      {submerge, swamp, every, tsunami, harbor, boat...   \n",
       "\n",
       "name                                             run_dry.v.01  \\\n",
       "adapted     {dry_out, run_dry, summer, river, empty, water...   \n",
       "offset-pos                                         02771756-v   \n",
       "original                           {empty, water, become, of}   \n",
       "simple      {dry_out, run_dry, summer, river, empty, water...   \n",
       "\n",
       "name                                              fog_up.v.01  \\\n",
       "adapted     {overcast, fog_up, foggy, fog, windshield, clo...   \n",
       "offset-pos                                         02771888-v   \n",
       "original                                         {foggy, get}   \n",
       "simple      {overcast, fog_up, foggy, fog, windshield, clo...   \n",
       "\n",
       "name                                                char.v.01  \\\n",
       "adapted     {coal, charcoal, forest, without, fire, everyt...   \n",
       "offset-pos                                         02771997-v   \n",
       "original                                 {charcoal, burn, to}   \n",
       "simple      {coal, charcoal, forest, without, fire, everyt...   \n",
       "\n",
       "name                                                haze.v.01  \\\n",
       "adapted     {dull, haze, cloud, become, overcast, hazy, cl...   \n",
       "offset-pos                                         02772202-v   \n",
       "original                  {dull, or, become, hazy, cloudy, ,}   \n",
       "simple      {dull, haze, cloud, become, overcast, hazy, cl...   \n",
       "\n",
       "name                                          deflagrate.v.01  \n",
       "adapted     {exercise, deflagrate, burn, substance, rapidl...  \n",
       "offset-pos                                         02772310-v  \n",
       "original    {to, and, with, burn, great, rapidly, intensit...  \n",
       "simple      {exercise, deflagrate, burn, substance, rapidl...  \n",
       "\n",
       "[4 rows x 117659 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pywsd_signatures = df.set_index('name').T\n",
    "pywsd_signatures.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pywsd_signatures.to_pickle('signatures.pkl',protocol=2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
